/* Orbas:
 *     A open source CORBA Specification implementation from Huihoo.
 *
 * Copyright 2002-2003 Huihoo.org, Inc. All Right Reserved.
 *
 * This software is licensed under LGPL license.
 * See terms of license at gnu.org.
 *
 * For more information, visit:
 *
 * http://www.huihoo.org/orbas
 */

package org.huihoo.orbas.idl;

import java.util.Hashtable;
import java.util.Stack;

import java_cup.runtime.char_token;
import java_cup.runtime.float_token;
import java_cup.runtime.int_token;
import java_cup.runtime.long_token;
import java_cup.runtime.token;

/**
 * This class implements a scanner (aka lexical analyzer or lexer) for IDL. The
 * scanner reads characters from a global input stream and returns integers
 * corresponding to the terminal number of the next token. Once the end of input
 * is reached the EOF token is returned on every subsequent call.
 * <p>
 * 
 * All symbol constants are defined in sym.java which is generated by JavaCup
 * from parser.cup.
 * <p>
 * 
 * In addition to the scanner proper (called first via init() then with
 * next_token() to get each token) this class provides simple error and warning
 * routines and keeps a count of errors and warnings that is publicly
 * accessible. It also provides basic preprocessing facilties, i.e. it does
 * handle preprocessor directives such as #define, #undef, #include, etc.
 * although it does not provide full C++ preprocessing
 * 
 * This class is "static" (i.e., it has only static members and methods).
 * 
 * @version $Id: lexer.java,v 1.2 2003/06/08 14:02:16 mep7591 Exp $
 * @author Gerald Brose
 * 
 */

public class lexer {

	/** First and second character of lookahead. */
	protected static int next_char;
	protected static int next_char2;

	/** EOF constant. */
	protected static final int EOF_CHAR = -1;

	/**
	 * Table of keywords. Keywords are initially treated as identifiers. Just
	 * before they are returned we look them up in this table to see if they
	 * match one of the keywords. The string of the name is the key here, which
	 * indexes Integer objects holding the symbol number.
	 */

	protected static Hashtable keywords = new Hashtable();

	/**
	 * Table of keywords, stored in lower case. Keys are the lower case version
	 * of the keywords used as keys for the keywords hash above, and the values
	 * are the case sensitive versions of the keywords. This table is used for
	 * detecting collisions of identifiers with keywords.
	 */
	protected static Hashtable keywords_lower_case = new Hashtable();

	/**
	 * Table of Java reserved names.
	 */

	protected static Hashtable java_keywords = new Hashtable();

	/**
	 * Table of single character symbols. For ease of implementation, we store
	 * all unambiguous single character tokens in this table of Integer objects
	 * keyed by Integer objects with the numerical value of the appropriate char
	 * (currently Character objects have a bug which precludes their use in
	 * tables).
	 */
	protected static Hashtable char_symbols = new Hashtable(25);

	/** Defined symbols (preprocessor) */

	protected static Hashtable defines = new Hashtable();
	protected static boolean conditionalCompilation = true;

	/** nested #ifdefs are pushed on this stack by the "preprocessor" */

	private static java.util.Stack ifStack = new Stack();
	private static java.util.Stack tokenStack = new Stack();

	/** Current line number for use in error messages. */
	protected static int current_line = 1;

	/** Current line for use in error messages. */
	protected static StringBuffer line = new StringBuffer();

	/** Character position in current line. */
	protected static int current_position = 1;

	/** Have we already read a '"' ? */
	protected static boolean in_string = false;

	/** Are we processing a wide char or string ? */
	protected static boolean wide = false;

	/** Count of total errors detected so far. */
	static int error_count = 0;

	/** Count of warnings issued so far */
	public static int warning_count = 0;

	/** currently active pragma prefix */
	public static String currentPragmaPrefix = "";

	/** reset the scanner state */

	public static void reset() {
		current_position = 1;
		error_count = 0;
		warning_count = 0;
		currentPragmaPrefix = "";
		line = new StringBuffer();
		ifStack.removeAllElements();
		tokenStack.removeAllElements();
		defines.clear();
	}

	/**
	 * Initialize the scanner. This sets up the keywords and char_symbols tables
	 * and reads the first two characters of lookahead.
	 * 
	 * "Object" is listed as reserved in the OMG spec. "int" is not, but I
	 * reserved it to bar its usage as a legal integer type.
	 */

	public static void init() throws java.io.IOException {
		/* set up standard symbols */
		defines.put("orbas_IDL_1_4", "");

		/* set up the keyword table */

		keywords.put("abstract", new Integer(sym.ABSTRACT));
		keywords.put("any", new Integer(sym.ANY));
		keywords.put("attribute", new Integer(sym.ATTRIBUTE));
		keywords.put("boolean", new Integer(sym.BOOLEAN));
		keywords.put("case", new Integer(sym.CASE));
		keywords.put("char", new Integer(sym.CHAR));
		keywords.put("const", new Integer(sym.CONST));
		keywords.put("context", new Integer(sym.CONTEXT));
		keywords.put("custom", new Integer(sym.CUSTOM));
		keywords.put("default", new Integer(sym.DEFAULT));
		keywords.put("double", new Integer(sym.DOUBLE));
		keywords.put("enum", new Integer(sym.ENUM));
		keywords.put("exception", new Integer(sym.EXCEPTION));
		keywords.put("factory", new Integer(sym.FACTORY));
		keywords.put("FALSE", new Integer(sym.FALSE));
		keywords.put("fixed", new Integer(sym.FIXED));
		keywords.put("float", new Integer(sym.FLOAT));
		keywords.put("in", new Integer(sym.IN));
		keywords.put("inout", new Integer(sym.INOUT));
		keywords.put("interface", new Integer(sym.INTERFACE));
		keywords.put("local", new Integer(sym.LOCAL));
		keywords.put("long", new Integer(sym.LONG));
		keywords.put("module", new Integer(sym.MODULE));
		keywords.put("native", new Integer(sym.NATIVE));
		keywords.put("Object", new Integer(sym.OBJECT));
		keywords.put("octet", new Integer(sym.OCTET));
		keywords.put("oneway", new Integer(sym.ONEWAY));
		keywords.put("out", new Integer(sym.OUT));
		keywords.put("private", new Integer(sym.PRIVATE));
		keywords.put("public", new Integer(sym.PUBLIC));
		keywords.put("pseudo", new Integer(sym.PSEUDO));
		keywords.put("raises", new Integer(sym.RAISES));
		keywords.put("readonly", new Integer(sym.READONLY));
		keywords.put("sequence", new Integer(sym.SEQUENCE));
		keywords.put("short", new Integer(sym.SHORT));
		keywords.put("string", new Integer(sym.STRING));
		keywords.put("struct", new Integer(sym.STRUCT));
		keywords.put("supports", new Integer(sym.SUPPORTS));
		keywords.put("switch", new Integer(sym.SWITCH));
		keywords.put("TRUE", new Integer(sym.TRUE));
		keywords.put("truncatable", new Integer(sym.TRUNCATABLE));
		keywords.put("typedef", new Integer(sym.TYPEDEF));
		keywords.put("unsigned", new Integer(sym.UNSIGNED));
		keywords.put("union", new Integer(sym.UNION));
		keywords.put("ValueBase", new Integer(sym.VALUEBASE));
		keywords.put("valuetype", new Integer(sym.VALUETYPE));
		keywords.put("void", new Integer(sym.VOID));
		keywords.put("wchar", new Integer(sym.WCHAR));
		keywords.put("wstring", new Integer(sym.WSTRING));

		keywords.put("int", new Integer(sym.INT));

		keywords.put("::", new Integer(sym.DBLCOLON));
		keywords.put("<<", new Integer(sym.LSHIFT));
		keywords.put(">>", new Integer(sym.RSHIFT));
		keywords.put("L\"", new Integer(sym.LDBLQUOTE));

		// setup the mapping of lower case keywords to case sensitive
		// keywords

		for (java.util.Enumeration e = keywords.keys(); e.hasMoreElements();) {
			String keyword = (String) e.nextElement();
			String keyword_lower_case = keyword.toLowerCase();
			keywords_lower_case.put(keyword_lower_case, keyword);
		}

		/* set up the table of single character symbols */
		char_symbols.put(new Integer(';'), new Integer(sym.SEMI));
		char_symbols.put(new Integer(','), new Integer(sym.COMMA));
		char_symbols.put(new Integer('*'), new Integer(sym.STAR));
		char_symbols.put(new Integer('.'), new Integer(sym.DOT));
		char_symbols.put(new Integer(':'), new Integer(sym.COLON));
		char_symbols.put(new Integer('='), new Integer(sym.EQUALS));
		char_symbols.put(new Integer('+'), new Integer(sym.PLUS));
		char_symbols.put(new Integer('-'), new Integer(sym.MINUS));
		char_symbols.put(new Integer('{'), new Integer(sym.LCBRACE));
		char_symbols.put(new Integer('}'), new Integer(sym.RCBRACE));
		char_symbols.put(new Integer('('), new Integer(sym.LPAREN));
		char_symbols.put(new Integer(')'), new Integer(sym.RPAREN));
		char_symbols.put(new Integer('['), new Integer(sym.LSBRACE));
		char_symbols.put(new Integer(']'), new Integer(sym.RSBRACE));
		char_symbols.put(new Integer('<'), new Integer(sym.LESSTHAN));
		char_symbols.put(new Integer('>'), new Integer(sym.GREATERTHAN));
		char_symbols.put(new Integer('\''), new Integer(sym.QUOTE));
		char_symbols.put(new Integer('\"'), new Integer(sym.DBLQUOTE));
		char_symbols.put(new Integer('\\'), new Integer(sym.BSLASH));
		char_symbols.put(new Integer('^'), new Integer(sym.CIRCUM));
		char_symbols.put(new Integer('&'), new Integer(sym.AMPERSAND));
		char_symbols.put(new Integer('/'), new Integer(sym.SLASH));
		char_symbols.put(new Integer('%'), new Integer(sym.PERCENT));
		char_symbols.put(new Integer('~'), new Integer(sym.TILDE));
		char_symbols.put(new Integer('|'), new Integer(sym.BAR));

		/* set up reserved Java names */

		java_keywords.put("abstract", "");
		java_keywords.put("boolean", "");
		java_keywords.put("break", "");
		java_keywords.put("byte", "");
		java_keywords.put("case", "");
		java_keywords.put("catch", "");
		java_keywords.put("char", "");
		java_keywords.put("class", "");
		java_keywords.put("const", "");
		java_keywords.put("continue", "");
		java_keywords.put("default", "");
		java_keywords.put("do", "");
		java_keywords.put("double", "");
		java_keywords.put("else", "");
		java_keywords.put("extends", "");
		java_keywords.put("false", "");
		java_keywords.put("final", "");
		java_keywords.put("finally", "");
		java_keywords.put("for", "");
		java_keywords.put("float", "");
		java_keywords.put("goto", "");
		java_keywords.put("if", "");
		java_keywords.put("implements", "");
		java_keywords.put("import", "");
		java_keywords.put("instanceof", "");
		java_keywords.put("int", "");
		java_keywords.put("interface", "");
		java_keywords.put("long", "");
		java_keywords.put("native", "");
		java_keywords.put("new", "");
		java_keywords.put("null", "");
		java_keywords.put("package", "");
		java_keywords.put("protected", "");
		java_keywords.put("private", "");
		java_keywords.put("public", "");
		java_keywords.put("return", "");
		java_keywords.put("short", "");
		java_keywords.put("static", "");
		java_keywords.put("super", "");
		java_keywords.put("switch", "");
		java_keywords.put("synchronized", "");
		java_keywords.put("true", "");
		java_keywords.put("this", "");
		java_keywords.put("throw", "");
		java_keywords.put("throws", "");
		java_keywords.put("transient", "");
		java_keywords.put("try", "");
		java_keywords.put("volatile", "");
		java_keywords.put("void", "");
		java_keywords.put("while", "");

		java_keywords.put("clone", "");
		java_keywords.put("equals", "");
		java_keywords.put("finalize", "");
		java_keywords.put("getClass", "");
		java_keywords.put("hashCode", "");
		java_keywords.put("notify", "");
		java_keywords.put("toString", "");
		java_keywords.put("wait", "");

		/* stack needs a topmost value */
		ifStack.push(new Boolean(true));

		/* read two characters of lookahead */

		try {
			next_char = GlobalInputStream.read();
		} catch (Exception e) {
			org.huihoo.orbas.idl.parser.fatal_error("Cannot read from file "
					+ GlobalInputStream.currentFile().getAbsolutePath()
					+ ", please check file name.", null);
		}

		if (next_char == EOF_CHAR)
			next_char2 = EOF_CHAR;
		else
			next_char2 = GlobalInputStream.read();
	}

	public static void define(String symbol, String value) {
		Environment.output(4, "Defining: " + symbol + " as " + value);
		defines.put(symbol, value);
	}

	public static void undefine(String symbol) {
		Environment.output(4, "Un-defining: " + symbol);
		defines.remove(symbol);
	}

	public static String defined(String symbol) {
		return (String) defines.get(symbol);
	}

	/**
	 * record information about the last lexical scope so that it can be
	 * restored later
	 */

	public static int currentLine() {
		return current_line;
	}

	/**
	 * asks the lexer to return its own
	 */

	public static PositionInfo getPosition() {
		return new PositionInfo(current_line, current_position,
				currentPragmaPrefix, line.toString());
	}

	public static void restorePosition(PositionInfo p) {
		current_line = p.line_no;
		currentPragmaPrefix = p.pragma_prefix;
		current_position = 0;
	}

	/**
	 * Advance the scanner one character in the input stream. This moves
	 * next_char2 to next_char and then reads a new next_char2.
	 */

	protected static void advance() throws java.io.IOException {
		int old_char;

		old_char = next_char;
		next_char = next_char2;
		next_char2 = GlobalInputStream.read();

		line.append((char) old_char);

		/* count this */

		current_position++;
		if (old_char == '\n') {
			current_line++;
			current_position = 1;
			line = new StringBuffer();
		}
		// System.out.println("advance. next_char is " + next_char + " (" +
		// (char)next_char + ")");
	}

	/**
	 * Emit an error message. The message will be marked with both the current
	 * line number and the position in the line. Error messages are printed on
	 * standard error (System.err).
	 * 
	 * @param message
	 *            the message to print.
	 * @param p_info
	 *            an optional PositionInfo object
	 */

	public static void emit_error(String message) {
		System.err.println(GlobalInputStream.currentFile().getAbsolutePath()
				+ ", line: " + current_line + "(" + current_position + "): "
				+ message);

		System.err.println("\t" + line.toString());
		error_count++;
	}

	public static void emit_error(String message, str_token t) {
		if (t == null) {
			emit_error(message);
		} else {
			System.err.println("Error in " + t.fileName + ", line:" + t.line_no
					+ "(" + t.char_pos + "): " + message);
			System.err.println("\t" + t.line_val);
			error_count++;
		}
	}

	/**
	 * Emit a warning message. The message will be marked with both the current
	 * line number and the position in the line. Messages are printed on
	 * standard error (System.err).
	 * 
	 * @param message
	 *            the message to print.
	 */

	public static void emit_warn(String message) {
		System.err.println("Warning: " + message + " at " + current_line + "("
				+ current_position + "): \"" + line.toString() + "\"");
		warning_count++;
	}

	public static void emit_warn(String message, str_token t) {
		if (t == null) {
			emit_warn(message);
		} else {
			System.err.println("Warning at " + t.fileName + ", line:"
					+ t.line_no + "(" + t.char_pos + "): " + message);
			System.err.println("\t" + t.line_val);
			warning_count++;
		}
	}

	/**
	 * Determine if a character is ok to start an id.
	 * 
	 * @param ch
	 *            the character in question.
	 */
	protected static boolean id_start_char(int ch) {
		return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')
				|| (ch == '_');
	}

	/**
	 * Determine if a character is ok for the middle of an id.
	 * 
	 * @param ch
	 *            the character in question.
	 */
	protected static boolean id_char(int ch) {
		return id_start_char(ch) || (ch == '_') || (ch >= '0' && ch <= '9');
	}

	/**
	 * Try to look up a single character symbol, returns -1 for not found.
	 * 
	 * @param ch
	 *            the character in question.
	 */

	protected static int find_single_char(int ch) {
		Integer result;

		result = (Integer) char_symbols.get(new Integer((char) ch));
		if (result == null)
			return -1;
		else
			return result.intValue();
	}

	/**
	 * Handle swallowing up a comment. Both old style C and new style C++
	 * comments are handled.
	 */
	protected static void swallow_comment() throws java.io.IOException {
		/* next_char == '/' at this point */

		/* is it a traditional comment */
		if (next_char2 == '*') {
			/* swallow the opener */
			advance();
			advance();

			/* swallow the comment until end of comment or EOF */
			for (;;) {
				/* if its EOF we have an error */
				if (next_char == EOF_CHAR) {
					emit_error("Specification file ends inside a comment", null);
					return;
				}

				/* if we can see the closer we are done */
				if (next_char == '*' && next_char2 == '/') {
					advance();
					advance();
					return;
				}

				/* otherwise swallow char and move on */
				advance();
			}
		}

		/* is its a new style comment */
		if (next_char2 == '/') {
			/* swallow the opener */
			advance();
			advance();

			/* swallow to '\n', '\f', or EOF */
			while (next_char != '\n' && next_char != '\f' && next_char != '\r'
					&& next_char != EOF_CHAR) {
				advance();
			}

			return;
		}

		/* shouldn't get here, but... if we get here we have an error */
		emit_error("Malformed comment in specification -- ignored", null);
		advance();
	}

	/**
	 * Preprocessor directives are handled here.
	 */

	protected static void preprocess() throws java.io.IOException {
		for (;;) {
			/* if its EOF we have an error */
			if (next_char == EOF_CHAR) {
				emit_error(
						"Specification file ends inside a preprocessor directive",
						null);
				return;
			} else if (next_char != '#') {
				emit_error("expected #, got " + (char) next_char + " instead!",
						null);
			} else
				advance(); // skip '#'

			// the following is done to allow for # ifdef sloppiness
			while ((' ' == next_char) || ('\t' == next_char))
				advance();

			String dir = get_string();

			if (dir.equals("include")) {
				if (!conditionalCompilation)
					return;
				advance(); // skip ' '
				boolean useIncludePath = (next_char == '<');
				advance(); // skip `\"' or '<'

				String fname = get_string();

				if (useIncludePath && (next_char != '>'))
					emit_error("Syntax error in #include directive, expecting '>'");
				else if (!useIncludePath && (next_char != '\"'))
					emit_error("Syntax error in #include directive, expecting \"");

				/* swallow to '\n', '\f', or EOF */
				while (next_char != '\n' && next_char != '\f'
						&& next_char != '\r' && next_char != EOF_CHAR) {
					advance();
				}
				GlobalInputStream.include(fname, (char) next_char2,
						useIncludePath);
				current_line = 0;
				advance();
				advance();
				// System.out.println("returning from include, next_char is: " +
				// (char)next_char);
				return;
			} else if (dir.equals("define")) {
				if (!conditionalCompilation)
					return;
				// advance(); // skip ' '
				swallow_whitespace();
				String name = get_string();
				StringBuffer text = new StringBuffer();
				if (next_char == ' ') {
					advance();
				}
				while (next_char != '\n') {
					if (next_char == '\\') {
						advance();
						advance();
					}
					text.append((char) next_char);
					advance();
				}
				// System.out.println("#Defined symbol " + name + " as: " +
				// text.toString() );
				define(name, text.toString());
			} else if (dir.equals("error")) {
				if (!conditionalCompilation)
					return;
				advance(); // skip ' '
				String name = get_string();
				emit_error(name);
			} else if (dir.equals("undef")) {
				if (!conditionalCompilation)
					return;
				swallow_whitespace(); // advance(); // skip ' '
				String name = get_string();
				undefine(name);
				// System.out.println("#Undefined symbol " + name );
			} else if (dir.equals("if")) {
				ifStack.push(new Boolean(conditionalCompilation));
				if (!conditionalCompilation)
					return;

				swallow_whitespace();

				// the following snippet distinguishes between #if defined
				// and #if !defined

				boolean straightDefined = true;
				if ('!' == next_char) {
					advance();
					straightDefined = false;
				}

				String defineStr = get_string_no_paren();
				if (!(defineStr.equals("defined"))) {
					emit_error("Expected \"defined\" following #if: " + dir,
							null);
					return;
				}

				swallow_whitespace();

				boolean brackets = ('(' == next_char);
				if (brackets) {
					advance(); // skip '('
					swallow_whitespace(); // any whitespace after ( ? skip it
				}

				String name = get_string_no_paren();

				if (brackets) {
					swallow_whitespace();
					Environment.output(4, "next char: " + next_char);

					if (')' != next_char) {
						emit_error("Expected ) terminating #if defined", null);
						return;
					}
					advance();
				}

				if (straightDefined)
					conditionalCompilation = (null != defined(name));
				else
					conditionalCompilation = (null == defined(name));
			} else if (dir.equals("ifdef")) {
				ifStack.push(new Boolean(conditionalCompilation));
				if (!conditionalCompilation)
					return;
				swallow_whitespace(); // advance(); // skip ' '
				String name = get_string();
				conditionalCompilation = (defined(name) != null);
			} else if (dir.equals("ifndef")) {
				ifStack.push(new Boolean(conditionalCompilation));
				if (!conditionalCompilation)
					return;
				swallow_whitespace();// advance(); // skip ' '
				String name = get_string();
				conditionalCompilation = (defined(name) == null);
			} else if (dir.equals("else")) {
				if (((Boolean) ifStack.peek()).booleanValue())
					conditionalCompilation = !conditionalCompilation;
			} else if (dir.equals("endif")) {
				boolean b = ((Boolean) ifStack.pop()).booleanValue();
				conditionalCompilation = b;
			} else if (dir.equals("pragma")) {
				if (!conditionalCompilation)
					return;
				swallow_whitespace();
				// advance(); // skip ' '
				String name = get_string();
				if (name.equals("prefix")) {
					advance(); // skip ' '
					// if( (char)next_char != '\"' )
					// emit_error("Expecting \" ");
					// advance(); // skip '"'
					String prefix = get_string();
					// if( (char)next_char != '\"' )
					// emit_error("Expecting \" ");
					// advance(); // skip '"'
					currentPragmaPrefix = prefix;
				} else if (name.equals("version")) {
					advance(); // skip ' '
					String vname = get_string();
					advance(); // skip ' '
					String version = get_string();
					String existingVersion = (String) parser.currentScopeData().versionMap
							.get(vname);
					if (existingVersion == null) {
						// Set version

						parser.currentScopeData().versionMap
								.put(vname, version);
					} else {
						// Check for version change

						if (!existingVersion.equals(version)) {
							emit_warn(
									"Version re-declaration, ignoring: #pragma version "
											+ version, null);
						}
					}
				} else if (name.equals("ID")) {
					advance(); // skip ' '
					String iname = get_string();
					advance(); // skip ' '
					String id = get_string();
					// do something with it
				} else if (name.equals("inhibit_code_generation")) {
					/* proprietary pragma of the orbas IDL compiler */
					parser.setInhibitionState(true);
					// do something with it
				} else {
					emit_warn("Unknown pragma, ignoring: #pragma " + name, null);
				}
			} else {
				emit_error("Unrecognized preprocessor directive " + dir, null);
			}

			/* swallow to '\n', '\f', or EOF */
			while (next_char != '\n' && next_char != '\f' && next_char != '\r'
					&& next_char != EOF_CHAR) {
				// System.out.println("Advancing after directive, next_char is: "
				// + (char)next_char);
				advance();
			}
			return;
		}
	}

	// the following is used for parsing the #if defined(...) construct

	private static String get_string_no_paren() throws java.io.IOException {
		StringBuffer sb = new StringBuffer();
		char c = (char) next_char;
		while (c != ' ' && c != '\t' && c != '\r' && c != '\n' && c != '\f'
				&& c != EOF_CHAR && c != '\"' && c != '<' && c != '>'
				&& c != '(' && c != ')') {
			sb.append(c);
			advance();
			c = (char) next_char;
		}
		// System.out.println("get string returns " + sb.toString());
		return sb.toString();
	}

	private static String get_string() throws java.io.IOException {

		StringBuffer sb = new StringBuffer("");

		if (next_char == '\"') {
			advance();
			while (next_char != '\"') {
				if (next_char == EOF_CHAR)
					emit_error("Unexpected EOF in string");
				sb.append((char) next_char);
				advance();
			}
		} else {
			while (next_char != ' ' && next_char != '\t' && next_char != '\r'
					&& next_char != '\n' && next_char != '\f'
					&& next_char != EOF_CHAR && next_char != '\"'
					&& next_char != '<' && next_char != '>') {
				sb.append((char) next_char);
				advance();
			}
		}
		return sb.toString();
	}

	/**
	 * Process an identifier.
	 * <P>
	 * Identifiers begin with a letter, underscore, or dollar sign, which is
	 * followed by zero or more letters, numbers, underscores or dollar signs.
	 * This routine returns a str_token suitable for return by the scanner or
	 * null, if the string that was read expanded to a symbol that was #defined.
	 * In this case, the symbol is expanded in place
	 */

	protected static token do_symbol() throws java.io.IOException {
		StringBuffer result = new StringBuffer();
		String result_str;
		Integer keyword_num = null;
		char buffer[] = new char[1];

		/* next_char holds first character of id */
		buffer[0] = (char) next_char;
		result.append(buffer, 0, 1);
		advance();

		/* collect up characters while they fit in id */
		while (id_char(next_char)) {
			buffer[0] = (char) next_char;
			result.append(buffer, 0, 1);
			advance();
		}

		/* extract a string */
		result_str = result.toString();

		/* try to look it up as a defined symbol... */

		String text = defined(result_str);
		if (text != null) {
			char[] next = { (char) next_char, (char) next_char2 };
			GlobalInputStream.insert(text + (new String(next)));
			// System.out.println("Advancing after symbol " + result_str );
			advance(); // restore lookahead
			advance(); // restore lookahead
			return null;
		}

		// check if it's a keyword
		Environment.output(3, "Advancing after symbol " + result_str);

		keyword_num = (Integer) keywords.get(result_str);
		if (keyword_num != null) {
			if (isScope(result_str)) {
				parser.openScope();
			}
			return new token(keyword_num.intValue());
		}

		// not a keyword, so treat as identifier after verifying
		// case sensitivity rules and prefacing with an _
		// if it collides with a Java keyword.

		result_str = checkIdentifier(result_str);
		if (null != result_str)
			return new str_token(sym.ID, result_str, getPosition(),
					GlobalInputStream.currentFile().getName());
		else
			return null;
	}

	private static boolean isScope(String keyword) {
		return (keyword.equals("module") || keyword.equals("interface")
				|| keyword.equals("struct") || keyword.equals("exception") || keyword
					.equals("union")
		// keyword.equals("valuetype")
		);
	}

	/**
	 * checks whether Identifier str is legal and returns it, prepends an
	 * underscore if necessary to avoid clashes with Java reserved words
	 */

	public static String checkIdentifier(String str) {
		Environment.output(3, "checking identifier " + str);

		/* if it is not an escaped identifier, look it up as a keyword */
		if (str.charAt(0) == '_') {
			return str.substring(1);
		}

		String colliding_keyword = (String) keywords_lower_case.get(str
				.toLowerCase());

		if (colliding_keyword != null) {
			emit_error("Identifier " + str + " collides with keyword "
					+ colliding_keyword + ".");
			return null;
		}

		if (needsJavaEscape(str)) {
			// Environment.output(4, "checking identifier " + str +
			// " : needs _ ");
			return "_" + str;
		}

		// Environment.output(4, "checking identifier " + str +
		// " : needs no _ ");
		return str;
	}

	/**
	 * Only the most general name clashes with Java keywords are caught here.
	 * Identifiers need to be checked again at different other places in the
	 * compiler!
	 */

	private static boolean needsJavaEscape(String s) {
		return (java_keywords.containsKey(s));
	}

	public static boolean strictJavaEscapeCheck(String s) {
		return ((!s.equals("Helper") && s.endsWith("Helper"))
				|| (!s.equals("Holder") && s.endsWith("Holder"))
				|| (!s.equals("Operations") && s.endsWith("Operations"))
				|| (!s.equals("Package") && s.endsWith("Package"))
				|| (!s.equals("POA") && s.endsWith("POA")) || (!s
				.equals("POATie") && s.endsWith("POATie")));
	}

	public static boolean needsJavaEscape(Module m) {
		String s = m.pack_name;
		Environment.output(4, "checking module name " + s);
		return (strictJavaEscapeCheck(s));
	}

	/**
	 * Return one token. This is the main external interface to the scanner. It
	 * consumes sufficient characters to determine the next input token and
	 * returns it.
	 */

	public static token next_token() throws java.io.IOException {
		parser.set_included(GlobalInputStream.includeState());
		token result = real_next_token();
		// System.out.println("# next_token() => " + result.sym);
		return result;
	}

	private static void swallow_whitespace() throws java.io.IOException {
		/* look for white space */
		while (next_char == ' ' || next_char == '\t' || next_char == '\n'
				|| next_char == '\f' || next_char == '\r') {
			/* advance past it and try the next character */
			// System.out.println("Swallowing whitespace: " + next_char +
			// (char)next_char);
			advance();
		}
	}

	/**
	 * The actual routine to return one token.
	 */

	protected static token real_next_token() throws java.io.IOException {
		int sym_num;

		if (!tokenStack.empty())
			return (token) tokenStack.pop();

		for (;;) {
			if (!in_string) {
				swallow_whitespace();

				/* look for preprocessor directives */
				if ((char) next_char == '#') {
					preprocess();
					continue;
				}

				/* look for a comment */
				if (next_char == '/'
						&& (next_char2 == '*' || next_char2 == '/')) {
					/* swallow then continue the scan */
					swallow_comment();
					continue;
				}

				if (!conditionalCompilation) {
					advance();
					if (next_char == EOF_CHAR) {
						emit_error("EOF in conditional compilation!", null);
						return null;
					} else
						continue;
				}

				/* look for COLON or DBLCOLON */
				if (next_char == ':') {
					if (next_char2 == ':') {
						advance();
						advance();
						return new token(sym.DBLCOLON);
					} else {
						advance();
						return new token(sym.COLON);
					}
				}

				/* leading L for wide strings */
				if (next_char == 'L' && next_char2 == '\"') {
					advance();
					advance();
					wide = true;
					in_string = true;
					return new token(sym.LDBLQUOTE);
				}

				/* look for Shifts */
				if (next_char == '<') {
					if (next_char2 == '<') {
						advance();
						advance();
						return new token(sym.LSHIFT);
					} else {
						advance();
						return new token(sym.LESSTHAN);
					}
				}
				if (next_char == '>') {
					if (next_char2 == '>') {
						advance();
						advance();
						return new token(sym.RSHIFT);
					} else {
						advance();
						return new token(sym.GREATERTHAN);
					}
				}

				/* leading 0: */
				/*
				 * Try to scan octal/hexadecimal numbers, might even find a
				 * float
				 */
				if (next_char == '0') {
					long l_val = 0;
					long l_val_old = 0;
					int radix = 8;
					int digit = 0;
					advance();

					if (next_char == '.') {
						StringBuffer f_string = new StringBuffer("0.");
						advance();

						while (next_char >= '0' && next_char <= '9') {
							f_string.append((char) next_char);
							advance();
						}

						float f_val = (new Float(f_string.toString()))
								.floatValue();
						return new float_token(sym.FLOAT_NUMBER, f_val);
					} else {
						// See if hexadecimal value

						if (next_char == 'x' || next_char == 'X') {
							advance();
							radix = 16;
						}

						StringBuffer val = new StringBuffer("0");
						digit = Character.digit((char) next_char, radix);
						while (digit != -1) {
							val.append((char) next_char);
							advance();
							digit = Character.digit((char) next_char, radix);
						}

						String str = val.toString();
						try {
							return new int_token(sym.NUMBER, Integer.parseInt(
									str, radix));
						} catch (NumberFormatException ex) {
							try {
								return new long_token(sym.LONG_NUMBER,
										Long.parseLong(str, radix));
							} catch (NumberFormatException ex2) {
								emit_error("Invalid octal/hex value:  " + str);
							}
						}
						return null;
					}
				}

				/* Try to scan integer, floating point or fixed point literals */

				if ((next_char >= '0' && next_char <= '9') || next_char == '.') {
					StringBuffer value = new StringBuffer();
					StringBuffer fraction = null;
					int exp = 0;

					/* read integer part */
					while (next_char >= '0' && next_char <= '9') {
						value.append((char) next_char);
						// System.out.println("Read integer part " +
						// value.toString());
						advance();
					}

					/* read fraction */
					if (next_char == '.') {
						fraction = new StringBuffer();
						advance();

						// System.out.println("Reading fraction part");

						while (next_char >= '0' && next_char <= '9') {
							fraction.append((char) next_char);
							advance();
						}
					}

					if (next_char == 'e' || next_char == 'E') {
						// System.out.println("Reading exponent");

						if (fraction == null)
							fraction = new StringBuffer();

						fraction.append('e');
						advance();
						if (next_char == '-' || next_char == '+') {
							fraction.append((char) next_char);
							advance();
						}

						while (next_char >= '0' && next_char <= '9') {
							fraction.append((char) next_char);
							advance();
						}

						if (fraction.length() == 1) {
							emit_error("Empty exponent in float/double.");
							continue;
						}

						return new float_token(sym.FLOAT_NUMBER, Float.valueOf(
								value.toString() + "." + fraction.toString())
								.floatValue());
					}

					if (next_char == 'd' || next_char == 'D') {
						advance();
						if (fraction == null)
							fraction = new StringBuffer();

						java.math.BigDecimal bi = new java.math.BigDecimal(
								value.toString() + "." + fraction.toString());
						return new fixed_token(sym.FIXED_NUMBER, bi);

					}

					if (fraction == null) {
						/* integer or long */

						token tok = null;
						String str = value.toString();

						try {
							tok = new int_token(sym.NUMBER,
									Integer.parseInt(str));
						} catch (NumberFormatException ex) {
							try {
								tok = new long_token(sym.LONG_NUMBER,
										Long.parseLong(str));
							} catch (NumberFormatException ex2) {
								emit_error("Invalid long value:  " + str);
							}
						}

						return tok;
					} else {
						try {
							float f = Float.valueOf(
									value.toString() + "."
											+ fraction.toString()).floatValue();
							return new float_token(sym.FLOAT_NUMBER, f);
						} catch (NumberFormatException nf) {
							emit_error("Unexpected symbol:  "
									+ value.toString() + "."
									+ fraction.toString());
						}
					}
				}

				/* look for a single character symbol */
				sym_num = find_single_char(next_char);

				if ((char) next_char == '\"') {
					in_string = true;
					advance();
					return new token(sym.DBLQUOTE);
				}

				if ((char) next_char == '\'') {
					advance();

					token t = null;

					if (next_char == '\\') {
						// Now need to process escaped character.

						advance();

						if (isDigit((char) next_char)) {
							// Octal character
							char octal1 = '0';
							char octal2 = '0';
							char octal3 = (char) next_char;

							if (isDigit((char) next_char2)) {
								advance();
								octal2 = octal3;
								octal3 = (char) next_char;

								if (isDigit((char) next_char2)) {
									advance();
									octal1 = octal2;
									octal2 = octal3;
									octal3 = (char) next_char;
								}
							}

							t = new char_token(sym.CH, (char) Integer.parseInt(
									new String(new char[] { octal1, octal2,
											octal3 }), 8));
						} else if ((char) next_char == 'x') {
							// Hexadecimal character
							advance();

							char hex1 = '0';
							char hex2 = (char) next_char;

							if (isHexLetterOrDigit((char) next_char2)) {
								advance();
								hex1 = hex2;
								hex2 = (char) next_char;
							} else if ((char) next_char2 != '\'') {
								emit_error("Illegal hex character");
								return null;
							}

							t = new char_token(sym.CH, (char) Integer.parseInt(
									new String(new char[] { hex1, hex2 }), 16));

						} else if ((char) next_char == 'u') {
							if (wide == false) {
								emit_error("Unicode characters are only legal with wide character");
								return null;
							} else {
								// Hexadecimal character
								advance();

								char uni1 = '0';
								char uni2 = '0';
								char uni3 = '0';
								char uni4 = (char) next_char;

								if (isHexLetterOrDigit((char) next_char2)) {
									advance();
									uni3 = uni4;
									uni4 = (char) next_char;

									if (isHexLetterOrDigit((char) next_char2)) {
										advance();
										uni2 = uni3;
										uni3 = uni4;
										uni4 = (char) next_char;

										if (isHexLetterOrDigit((char) next_char2)) {
											advance();
											uni1 = uni2;
											uni2 = uni3;
											uni3 = uni4;
											uni4 = (char) next_char;
										} else if ((char) next_char2 != '\'') {
											emit_error("Illegal unicode character");
											return null;
										}
									} else if ((char) next_char2 != '\'') {
										emit_error("Illegal unicode character");
										return null;
									}
								} else if ((char) next_char2 != '\'') {
									emit_error("Illegal unicode character");
									return null;
								}

								t = new char_token(sym.CH,
										(char) Integer.parseInt(new String(
												new char[] { uni1, uni2, uni3,
														uni4 }), 16));
							}
						} else {
							switch (next_char) {
							case 'n': {
								t = new char_token(sym.CH, '\n');
								break;
							}
							case 't': {
								t = new char_token(sym.CH, '\t');
								break;
							}
							case 'v': {
								t = new char_token(sym.CH, '\013');
								break;
							}
							case 'b': {
								t = new char_token(sym.CH, '\b');
								break;
							}
							case 'r': {
								t = new char_token(sym.CH, '\r');
								break;
							}
							case 'f': {
								t = new char_token(sym.CH, '\f');
								break;
							}
							case 'a': {
								t = new char_token(sym.CH, '\007');
								break;
							}
							case '\\': {
								t = new char_token(sym.CH, '\\');
								break;
							}
							case '?': {
								t = new char_token(sym.CH, '?');
								break;
							}
							case '0': {
								t = new char_token(sym.CH, '\0');
								break;
							}
							case '\'': {
								t = new char_token(sym.CH, '\'');
								break;
							}
							case '\"': {
								t = new char_token(sym.CH, '\"');
								break;
							}
							default: {
								emit_error("Invalid escape symbol \'");
								return null;
							}
							}
						}
					} else {
						t = new char_token(sym.CH, (char) next_char);
					}
					advance();

					if ((char) next_char == '\'') {
						tokenStack.push(new token(sym.QUOTE));
						tokenStack.push(t);
						advance();
					} else {
						emit_error("Expecting closing \'");
						return null;
					}
					wide = false;

					return new token(sym.QUOTE);
				}

				if (sym_num != -1) {
					/* found one -- advance past it and return a token for it */
					advance();

					return new token(sym_num);
				}

				/* look for an id or keyword */
				if (id_start_char(next_char)) {
					token t = do_symbol();
					if (t != null)
						return t;
					else
						continue;
				}

				/* look for EOF */
				if (next_char == EOF_CHAR) {
					return new token(sym.EOF);
				}
			} else // in_string
			{

				/* empty string ? */
				if ((char) next_char == '\"') {
					in_string = false;
					advance();
					return new token(sym.DBLQUOTE);
					// return new org.huihoo.orbas.idl.str_token(sym.ID, "",
					// getPosition());
				}

				StringBuffer result = new StringBuffer();
				char previous = ' ';

				/* collect up characters while they fit in id */
				while (true) {
					if (next_char == '\\') {
						// Remap those characters that have no equivilant in
						// java
						switch (next_char2) {
						case 'a': {
							result.append("\\007");
							previous = 'a';
							advance();
							break;
						}
						case 'v': {
							result.append("\\013");
							previous = 'v';
							advance();
							break;
						}
						case '?': {
							result.append("?");
							previous = '?';
							advance();
							break;
						}
						// Replace \xA0 by octal equivilant
						case 'x': {
							advance();
							advance();

							// Now next_char will be A and next_char2 will be 0
							String octal = Integer.toOctalString(Integer
									.parseInt(new String(
											new char[] { (char) next_char,
													(char) next_char2 }), 16));
							if (octal.length() != 3) {
								if (octal.length() == 1) {
									octal = "0" + octal;
								}
								octal = "0" + octal;
							}
							result.append("\\" + octal);
							previous = (char) next_char2;

							advance();
							break;
						}
						case 'u': {
							if (wide == false) {
								emit_error("Unicode characters are only legal with wide strings");
								return null;
							} else {
								result.append((char) next_char);
								result.append((char) next_char2);
								advance();
								advance();

								char uni1 = (char) next_char;
								char uni2 = '0';
								char uni3 = '0';
								char uni4 = '0';

								if (isHexLetterOrDigit((char) next_char2)) {
									advance();
									uni2 = (char) next_char;

									if (isHexLetterOrDigit((char) next_char2)) {
										advance();
										uni3 = (char) next_char;

										if (isHexLetterOrDigit((char) next_char2)) {
											advance();
											uni4 = (char) next_char;
										} else {
											emit_error("Illegal unicode character");
											return null;
										}
									} else {
										emit_error("Illegal unicode character");
										return null;
									}
								} else {
									emit_error("Illegal unicode character");
									return null;
								}

								previous = uni4;
								result.append(uni1);
								result.append(uni2);
								result.append(uni3);
								result.append(uni4);
							}
							break;
						}
						default: {
							previous = (char) next_char;
							result.append((char) next_char);
						}
						}
					} else {
						previous = (char) next_char;
						result.append((char) next_char);
					}
					advance();

					// Handle backslash quote but exit if just quote
					if (((char) next_char) == '\"' && previous != '\\') {
						break;
					}
				}
				wide = false;

				// String s = checkIdentifier( result.toString() );
				String s = result.toString();

				/* build and return an id token with an attached string */
				return new org.huihoo.orbas.idl.str_token(sym.ID, s,
						getPosition(), GlobalInputStream.currentFile()
								.getName());
			}

			/* if we get here, we have an unrecognized character */
			emit_warn("Unrecognized character '"
					+ new Character((char) next_char) + "'(" + next_char
					+ ") -- ignored");

			/* advance past it */
			advance();
		}
	}

	/**
	 * Returns true if character is US ASCII 0-9
	 * 
	 * @param c
	 *            a value of type 'char'
	 * @return a value of type 'boolean'
	 */
	private static boolean isDigit(char c) {
		boolean result = false;

		if (c >= '\u0030') {
			if (c <= '\u0039') {
				// Range 0030 [0] -> 0039 [9]
				result = true;
			}
		}
		return result;
	}

	/**
	 * Returns true if character is US ASCII 0-9, a-f, A-F
	 * 
	 * @param c
	 *            a value of type 'char'
	 * @return a value of type 'boolean'
	 */
	private static boolean isHexLetterOrDigit(char c) {
		boolean result = false;

		if (c >= '\u0030') {
			if (c <= '\u0039') {
				// Range 0030 [0] -> 0039 [9]
				result = true;
			} else {
				if (c >= '\u0041') {
					if (c <= '\u0046') {
						// Range 0041 [A] -> 0046 [F]
						result = true;
					}
					if (c >= '\u0061') {
						if (c <= '\u0066') {
							// Range 0061 [a] -> 0066 [f]
							result = true;
						}
					}
				}
			}
		}
		return result;
	}
}
